{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "print(tf.__version__)\n",
    "print(tf.keras.__version__)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential()\n",
    "model.add(layers.Dense(32,activation='relu'))\n",
    "model.add(layers.Dense(32,activation='relu'))\n",
    "model.add(layers.Dense(10,activation='softmax'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "layers.Dense(32,activation='sigmoid')\n",
    "layers.Dense(32,activation=tf.sigmoid)\n",
    "layers.Dense(32,kernel_initializer='orthogonal')\n",
    "layers.Dense(32,kernel_initializer=tf.keras.initializers.glorot_normal)\n",
    "layers.Dense(32,kernel_regularizer=tf.keras.regularizers.l2(0.01))\n",
    "layers.Dense(32,kernel_regularizer=tf.keras.regularizers.l1(0.01))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential()\n",
    "model.add(layers.Dense(32,activation='relu'))\n",
    "model.add(layers.Dense(32,activation='relu'))\n",
    "model.add(layers.Dense(10,activation='softmax'))\n",
    "model.compile(\n",
    "    optimizer=tf.keras.Adam(0.001),\n",
    "    loss = tf.keras.losses.categorical_crossentropy,\n",
    "    metrics=[tf.keras.metrics.categorical_accuracy]\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_x = np.random.random((1000,72))\n",
    "train_y = np.random.random((1000,10))\n",
    "\n",
    "val_x = np.random.random((200,72))\n",
    "val_y = np.random.random((200,10))\n",
    "\n",
    "model.fit(train_x,train_y,epochs=10,batch_size=100,\n",
    "          validation_data=(val_x,val_y))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((train_x,train_y))\n",
    "dataset = dataset.batch(32)\n",
    "dataset = dataset.repeat()\n",
    "\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((val_x,val_y))\n",
    "val_dataset = val_dataset.batch(32)\n",
    "val_dataset = val_dataset.repeat()\n",
    "\n",
    "model.fit(dataset,epochs=10,steps_per_epoch=30,\n",
    "          validation_data=val_dataset,validation_steps=3)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "test_x = np.random.random((1000,72))\n",
    "test_y = np.random.random((1000,10))\n",
    "model.evaluate(test_x,test_y,batch_size=32)\n",
    "test_data = tf.data.Dataset.from_tensor_slices(((test_x,test_y)))\n",
    "test_data = test_data.batch(32).repeat()\n",
    "model.evaluate(test_data,steps=30)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "result = model.predict(test_x,batch_size=32)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "input_x = tf.keras.Input(shape=(72,))\n",
    "hidden1 = layers.Dense(32,activation='relu')(input_x)\n",
    "hidden2 = layers.Dense(16,activation='relu')(hidden1)\n",
    "pred = layers.Dense(10,activation='softmax')(hidden2)\n",
    "\n",
    "model = tf.keras.Model(inputs=input_x,outputs=pred)\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(0.01),\n",
    "              loss=tf.keras.losses.categorical_crossentropy,\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_x,train_y,batch_size=32,epochs=5)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class MyModel(tf.keras.Model):\n",
    "    def __init(self,num_classes=10):\n",
    "        super(MyModel,self).__ini__(name='my_modele')\n",
    "        self.num_classes = num_classes\n",
    "        self.layer1=layers.Dense(32,activation='relu')\n",
    "        self.layer2=layers.Dense(num_classes,activation='softmax')\n",
    "    \n",
    "    def call(self,inputs):\n",
    "        h1 = self.layer1(inputs)\n",
    "        out = self.layer2(h1)\n",
    "        return out\n",
    "    \n",
    "    def compute_output_shape(self,input_shape):\n",
    "        shape = tf.TensorShape(input_shape).as_list()\n",
    "        shape[-1] = self.num_classes\n",
    "        return tf.TensorShape(shape)\n",
    "    \n",
    "model = MyModel(num_classes=10)\n",
    "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
    "              loss=tf.keras.losses.categorical_crossentropy,\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_x,train_y,batch_size=16,epochs=5)\n",
    "\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class MyLayer(layers.Layer):\n",
    "    def __init__(self,output_dim,**kwargs):\n",
    "        self.output_dim = output_dim\n",
    "        super(MyLayer,self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self,input_shape):\n",
    "        shape = tf.TensorShape((input_shape[1],self.output_dim))\n",
    "        self.kernel = self.add_weight(name='kernel1',shape=shape,\n",
    "                                      initializer='uniform',trainable=True)\n",
    "        super(MyLayer,self).build(input_shape)\n",
    "    \n",
    "    def call(self,inputs):\n",
    "        return tf.matmul(inputs,self.kernel)\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        shape = tf.TensorShape(input_shape).as_list()\n",
    "        shape[-1] = self.output_dim\n",
    "        return tf.TensorShape(shape)\n",
    "    \n",
    "    def get_config(self):\n",
    "        base_config = super(MyLayer,self).get_config()\n",
    "        base_config['output_dim'] = self.output_dim\n",
    "        return base_config\n",
    "    \n",
    "    @classmethod\n",
    "    def from_config(cls,config):\n",
    "        return cls(**config)\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "    MyLayer(10),\n",
    "    layers.Activation('softmax')\n",
    "])\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
    "              loss=tf.keras.losses.categorical_crossentropy,\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_x,train_y,batch_size=16,epochs=5)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "callbacks=[\n",
    "    tf.keras.callbacks.EarlyStopping(patience=2,monitor='val_loss'),\n",
    "    tf.keras.callbacks.TensorBoard(log_dir='output/logs')\n",
    "]\n",
    "model.fit(train_x,train_y,batch_size=16,epochs=5,callbacks=callbacks,\n",
    "          validation_data=(val_x,val_y))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    layers.Dense(64,activation='relu',input_shape=(32,)),\n",
    "    layers.Dense(10,activation='softmax')\n",
    "])\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(0.001),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model.save_weights('output/weights/model')\n",
    "model.load_weights('output/weights/model')\n",
    "\n",
    "model.save_weights('output/model.h5',save_format='h5')\n",
    "model.load_weights('output/model.h5')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import json\n",
    "import pprint\n",
    "json_str = model.to_json()\n",
    "pprint.pprint(json.loads(json_str))\n",
    "fresh_model = tf.keras.models.model_from_json(json_str)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "yaml_str = model.to_yaml()\n",
    "print(yaml_str)\n",
    "fresh_model = tf.keras.models.model_from_yaml(yaml_str)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    layers.Dense(10,activation='softmax',input_shape=(72,)),\n",
    "    layers.Dense(10,activation='softmax')\n",
    "])\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(train_x,train_y,batch_size=32,epochs=5)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    layers.Dense(10,activation='softmax',input_shape=(72,)),\n",
    "    layers.Dense(10,activation='softmax')\n",
    "])\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_x,train_y,batch_size=32,epochs=5)\n",
    "\n",
    "model.save('output/all_model.h5')\n",
    "model = tf.keras.models.load_model('output/all_model.h5')\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    layers.Dense(10,activation='softmax'),\n",
    "    layers.Dense(10,activation='softmax')\n",
    "])\n",
    "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "estimator = tf.keras.estimator.model_to_estimator(model)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "strategy = tf.distribute.MirroredStrategy()\n",
    "with strategy.scope():\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(layers.Dense(16,activation='relu',input_shape=(10,)))\n",
    "    model.add(layers.Dense(1,activation='sigmoid'))\n",
    "    optimizer = tf.keras.optimizers.SGD(0.2)\n",
    "    model.compile(loss='binary_crossentroy',optimizer=optimizer)\n",
    "model.summary()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "x = np.random.random((1024,10))\n",
    "y = np.random.randint(2,size=(1024,1))\n",
    "x = tf.cast(x,tf.float32)\n",
    "dataset = tf.data.Dataset.from_tensor_slices((x,y))\n",
    "dataset = dataset.shuffle(buffer_size=1024).batch(32)\n",
    "model.fit(dataset,epochs=1)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}